{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys, os, _pickle as pickle\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import nltk\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "data_dir = '../data'\n",
    "ckpt_dir = '../checkpoint'\n",
    "word_embd_dir = '../checkpoint/word_embd'\n",
    "model_dir = '../checkpoint/model2v1'\n",
    "\n",
    "word_embd_dim = 100\n",
    "pos_embd_dim = 25\n",
    "dep_embd_dim = 25\n",
    "word_vocab_size = 400001\n",
    "pos_vocab_size = 10\n",
    "dep_vocab_size = 21\n",
    "relation_classes = 19\n",
    "state_size = 100\n",
    "batch_size = 10\n",
    "channels = 3\n",
    "lambda_l2 = 0.0001\n",
    "max_len_path = 70\n",
    "starter_learning_rate = 0.001\n",
    "decay_steps = 2000\n",
    "decay_rate = 0.96"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"input\"):\n",
    "    path_length = tf.placeholder(tf.int32, shape=[batch_size], name=\"path1_length\")\n",
    "    word_ids = tf.placeholder(tf.int32, shape=[batch_size, max_len_path], name=\"word_ids\")\n",
    "    pos_ids = tf.placeholder(tf.int32, [batch_size, max_len_path], name=\"pos_ids\")\n",
    "    dep_ids = tf.placeholder(tf.int32, [batch_size, max_len_path], name=\"dep_ids\")\n",
    "    y = tf.placeholder(tf.int32, [batch_size], name=\"y\")\n",
    "\n",
    "with tf.name_scope(\"word_embedding\"):\n",
    "    W = tf.Variable(tf.constant(0.0, shape=[word_vocab_size, word_embd_dim]), name=\"W\")\n",
    "    embedding_placeholder = tf.placeholder(tf.float32,[word_vocab_size, word_embd_dim])\n",
    "    embedding_init = W.assign(embedding_placeholder)\n",
    "    embedded_word = tf.nn.embedding_lookup(W, word_ids)\n",
    "    word_embedding_saver = tf.train.Saver({\"word_embedding/W\": W})\n",
    "\n",
    "with tf.name_scope(\"pos_embedding\"):\n",
    "    W = tf.Variable(tf.random_uniform([pos_vocab_size, pos_embd_dim]), name=\"W\")\n",
    "    embedded_pos = tf.nn.embedding_lookup(W, pos_ids)\n",
    "    pos_embedding_saver = tf.train.Saver({\"pos_embedding/W\": W})\n",
    "\n",
    "with tf.name_scope(\"dep_embedding\"):\n",
    "    W = tf.Variable(tf.random_uniform([dep_vocab_size, dep_embd_dim]), name=\"W\")\n",
    "    embedded_dep = tf.nn.embedding_lookup(W, dep_ids)\n",
    "    dep_embedding_saver = tf.train.Saver({\"dep_embedding/W\": W})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.variable_scope(\"word_lstm\"):\n",
    "    cell = tf.contrib.rnn.BasicLSTMCell(state_size)\n",
    "    state_series, current_state = tf.nn.dynamic_rnn(cell, embedded_word, sequence_length=path_length, dtype=tf.float32)\n",
    "    state_series_word = tf.reduce_max(state_series, axis=1)\n",
    "\n",
    "with tf.variable_scope(\"pos_lstm\"):\n",
    "    cell = tf.contrib.rnn.BasicLSTMCell(state_size)\n",
    "    state_series, current_state = tf.nn.dynamic_rnn(cell, embedded_pos, sequence_length=path_length, dtype=tf.float32)\n",
    "    state_series_pos = tf.reduce_max(state_series, axis=1)\n",
    "\n",
    "with tf.variable_scope(\"dep_lstm\"):\n",
    "    cell = tf.contrib.rnn.BasicLSTMCell(state_size)\n",
    "    state_series, current_state = tf.nn.dynamic_rnn(cell, embedded_dep, sequence_length=path_length, dtype=tf.float32)\n",
    "    state_series_dep = tf.reduce_max(state_series, axis=1)\n",
    "    \n",
    "state_series = tf.concat([state_series_word, state_series_pos, state_series_dep], 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'concat:0' shape=(10, 300) dtype=float32>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state_series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"hidden_layer\"):\n",
    "    W = tf.Variable(tf.truncated_normal([300, 100], -0.1, 0.1), name=\"W\")\n",
    "    b = tf.Variable(tf.zeros([100]), name=\"b\")\n",
    "    y_hidden_layer = tf.nn.relu(tf.matmul(state_series, W) + b)\n",
    "\n",
    "with tf.name_scope(\"dropout\"):\n",
    "    y_hidden_layer_drop = tf.nn.dropout(y_hidden_layer, 0.3)\n",
    "\n",
    "with tf.name_scope(\"softmax_layer\"):\n",
    "    W = tf.Variable(tf.truncated_normal([100, relation_classes], -0.1, 0.1), name=\"W\")\n",
    "    b = tf.Variable(tf.zeros([relation_classes]), name=\"b\")\n",
    "    logits = tf.matmul(y_hidden_layer_drop, W) + b\n",
    "    predictions = tf.argmax(logits, 1)\n",
    "\n",
    "tv_all = tf.trainable_variables()\n",
    "tv_regu = []\n",
    "non_reg = [\"word_embedding/W:0\",\"pos_embedding/W:0\",'dep_embedding/W:0',\"global_step:0\",'hidden_layer/b:0','softmax_layer/b:0']\n",
    "for t in tv_all:\n",
    "    if t.name not in non_reg:\n",
    "        if(t.name.find('biases')==-1):\n",
    "            tv_regu.append(t)\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    l2_loss = lambda_l2 * tf.reduce_sum([ tf.nn.l2_loss(v) for v in tv_regu ])\n",
    "    loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y))\n",
    "    total_loss = loss + l2_loss\n",
    "\n",
    "global_step = tf.Variable(0, trainable=False, name=\"global_step\")\n",
    "\n",
    "learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step, decay_steps, decay_rate, staircase=True)\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate).minimize(total_loss, global_step=global_step)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = open(data_dir + '/vocab.pkl', 'rb')\n",
    "vocab = pickle.load(f)\n",
    "f.close()\n",
    "\n",
    "word2id = dict((w, i) for i,w in enumerate(vocab))\n",
    "id2word = dict((i, w) for i,w in enumerate(vocab))\n",
    "\n",
    "unknown_token = \"UNKNOWN_TOKEN\"\n",
    "word2id[unknown_token] = word_vocab_size -1\n",
    "id2word[word_vocab_size-1] = unknown_token\n",
    "\n",
    "pos_tags_vocab = []\n",
    "for line in open(data_dir + '/pos_tags.txt'):\n",
    "        pos_tags_vocab.append(line.strip())\n",
    "\n",
    "dep_vocab = []\n",
    "for line in open(data_dir + '/dependency_types.txt'):\n",
    "    dep_vocab.append(line.strip())\n",
    "\n",
    "relation_vocab = []\n",
    "for line in open(data_dir + '/relation_types.txt'):\n",
    "    relation_vocab.append(line.strip())\n",
    "\n",
    "rel2id = dict((w, i) for i,w in enumerate(relation_vocab))\n",
    "id2rel = dict((i, w) for i,w in enumerate(relation_vocab))\n",
    "\n",
    "pos_tag2id = dict((w, i) for i,w in enumerate(pos_tags_vocab))\n",
    "id2pos_tag = dict((i, w) for i,w in enumerate(pos_tags_vocab))\n",
    "\n",
    "dep2id = dict((w, i) for i,w in enumerate(dep_vocab))\n",
    "id2dep = dict((i, w) for i,w in enumerate(dep_vocab))\n",
    "\n",
    "pos_tag2id['OTH'] = 9\n",
    "id2pos_tag[9] = 'OTH'\n",
    "\n",
    "dep2id['OTH'] = 20\n",
    "id2dep[20] = 'OTH'\n",
    "\n",
    "JJ_pos_tags = ['JJ', 'JJR', 'JJS']\n",
    "NN_pos_tags = ['NN', 'NNS', 'NNP', 'NNPS']\n",
    "RB_pos_tags = ['RB', 'RBR', 'RBS']\n",
    "PRP_pos_tags = ['PRP', 'PRP$']\n",
    "VB_pos_tags = ['VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ']\n",
    "_pos_tags = ['CC', 'CD', 'DT', 'IN']\n",
    "\n",
    "def pos_tag(x):\n",
    "    if x in JJ_pos_tags:\n",
    "        return pos_tag2id['JJ']\n",
    "    if x in NN_pos_tags:\n",
    "        return pos_tag2id['NN']\n",
    "    if x in RB_pos_tags:\n",
    "        return pos_tag2id['RB']\n",
    "    if x in PRP_pos_tags:\n",
    "        return pos_tag2id['PRP']\n",
    "    if x in VB_pos_tags:\n",
    "        return pos_tag2id['VB']\n",
    "    if x in _pos_tags:\n",
    "        return pos_tag2id[x]\n",
    "    else:\n",
    "        return 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# f = open('data/word_embedding', 'rb')\n",
    "# word_embedding = pickle.load(f)\n",
    "# f.close()\n",
    "# sess.run(embedding_init, feed_dict={embedding_placeholder:word_embedding})\n",
    "# word_embedding_saver.save(sess, word_embd_dir + '/word_embd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# model = tf.train.latest_checkpoint(model_dir)\n",
    "# saver.restore(sess, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from checkpoint/word_embd/word_embd\n"
     ]
    }
   ],
   "source": [
    "latest_embd = tf.train.latest_checkpoint(word_embd_dir)\n",
    "word_embedding_saver.restore(sess, latest_embd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = open(data_dir + '/train_lca_paths', 'rb')\n",
    "word_p, dep_p, pos_p = pickle.load(f)\n",
    "f.close()\n",
    "relations = []\n",
    "for line in open(data_dir + '/train_relations.txt'):\n",
    "    relations.append(line.strip().split()[1])\n",
    "\n",
    "length = len(word_p)\n",
    "num_batches = int(length/batch_size)\n",
    "\n",
    "for i in range(length):\n",
    "    for j, word in enumerate(word_p[i]):\n",
    "        word = word.lower()\n",
    "        word_p[i][j] = word if word in word2id else unknown_token \n",
    "    for l, d in enumerate(dep_p[i]):\n",
    "        dep_p[i][l] = d if d in dep2id else 'OTH'\n",
    "        \n",
    "word_p_ids = np.ones([length, max_len_path],dtype=int)\n",
    "pos_p_ids = np.ones([length, max_len_path],dtype=int)\n",
    "dep_p_ids = np.ones([length, max_len_path],dtype=int)\n",
    "rel_ids = np.array([rel2id[rel] for rel in relations])\n",
    "path_len = np.array([len(w) for w in word_p], dtype=int)\n",
    "\n",
    "for i in range(length):\n",
    "    for j, w in enumerate(word_p[i]):\n",
    "        word_p_ids[i][j] = word2id[w]\n",
    "        \n",
    "    for j, w in enumerate(pos_p[i]):\n",
    "        pos_p_ids[i][j] = pos_tag(w)\n",
    "        \n",
    "    for j, w in enumerate(dep_p[i]):\n",
    "        dep_p_ids[i][j] = dep2id[w]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 Step: 800 loss: 2.85355205297\n",
      "Saved Model\n",
      "Epoch: 2 Step: 1600 loss: 2.73827668965\n",
      "Saved Model\n",
      "Epoch: 3 Step: 2400 loss: 2.70001435518\n",
      "Saved Model\n",
      "Epoch: 4 Step: 3200 loss: 2.68624746531\n",
      "Saved Model\n",
      "Epoch: 5 Step: 4000 loss: 2.68042603165\n",
      "Saved Model\n",
      "Epoch: 6 Step: 4800 loss: 2.67750604913\n",
      "Saved Model\n",
      "Epoch: 7 Step: 5600 loss: 2.67583220631\n",
      "Saved Model\n",
      "Epoch: 8 Step: 6400 loss: 2.67482194766\n",
      "Saved Model\n",
      "Epoch: 9 Step: 7200 loss: 2.67411908716\n",
      "Saved Model\n",
      "Epoch: 10 Step: 8000 loss: 2.67369878128\n",
      "Saved Model\n",
      "Epoch: 11 Step: 8800 loss: 2.67341704309\n",
      "Saved Model\n",
      "Epoch: 12 Step: 9600 loss: 2.67321884066\n",
      "Saved Model\n",
      "Epoch: 13 Step: 10400 loss: 2.67310401961\n",
      "Saved Model\n",
      "Epoch: 14 Step: 11200 loss: 2.67295600712\n",
      "Saved Model\n",
      "Epoch: 15 Step: 12000 loss: 2.67288722694\n",
      "Saved Model\n",
      "Epoch: 16 Step: 12800 loss: 2.67282888472\n",
      "Saved Model\n",
      "Epoch: 17 Step: 13600 loss: 2.67277920395\n",
      "Saved Model\n",
      "Epoch: 18 Step: 14400 loss: 2.6727619794\n",
      "Saved Model\n",
      "Epoch: 19 Step: 15200 loss: 2.67268569678\n",
      "Saved Model\n",
      "Epoch: 20 Step: 16000 loss: 2.67266457796\n",
      "Saved Model\n",
      "Epoch: 21 Step: 16800 loss: 2.67263956338\n",
      "Saved Model\n",
      "Epoch: 22 Step: 17600 loss: 2.67261722207\n",
      "Saved Model\n",
      "Epoch: 23 Step: 18400 loss: 2.67261824235\n",
      "Saved Model\n",
      "Epoch: 24 Step: 19200 loss: 2.67256126881\n",
      "Saved Model\n",
      "Epoch: 25 Step: 20000 loss: 2.6725519672\n",
      "Saved Model\n",
      "Epoch: 26 Step: 20800 loss: 2.67253558069\n",
      "Saved Model\n",
      "Epoch: 27 Step: 21600 loss: 2.67252239197\n",
      "Saved Model\n",
      "Epoch: 28 Step: 22400 loss: 2.67252858594\n",
      "Saved Model\n",
      "Epoch: 29 Step: 23200 loss: 2.67248077154\n",
      "Saved Model\n",
      "Epoch: 30 Step: 24000 loss: 2.67247578681\n",
      "Saved Model\n",
      "Epoch: 31 Step: 24800 loss: 2.67246250227\n",
      "Saved Model\n",
      "Epoch: 32 Step: 25600 loss: 2.67245363146\n",
      "Saved Model\n",
      "Epoch: 33 Step: 26400 loss: 2.67246143714\n",
      "Saved Model\n",
      "Epoch: 34 Step: 27200 loss: 2.6724195759\n",
      "Saved Model\n",
      "Epoch: 35 Step: 28000 loss: 2.67241657913\n",
      "Saved Model\n",
      "Epoch: 36 Step: 28800 loss: 2.67240460932\n",
      "Saved Model\n",
      "Epoch: 37 Step: 29600 loss: 2.67239822775\n",
      "Saved Model\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 40\n",
    "for i in range(num_epochs):\n",
    "    loss_per_epoch = 0\n",
    "    for j in range(num_batches):\n",
    "        feed_dict = {\n",
    "            path_length:path_len[j*batch_size:(j+1)*batch_size],\n",
    "            word_ids:word_p_ids[j*batch_size:(j+1)*batch_size],\n",
    "            pos_ids:pos_p_ids[j*batch_size:(j+1)*batch_size],\n",
    "            dep_ids:dep_p_ids[j*batch_size:(j+1)*batch_size],\n",
    "            y:rel_ids[j*batch_size:(j+1)*batch_size]}\n",
    "        _, _loss, step = sess.run([optimizer, total_loss, global_step], feed_dict)\n",
    "        loss_per_epoch +=_loss\n",
    "        if (j+1)%num_batches==0:\n",
    "            print(\"Epoch:\", i+1,\"Step:\", step, \"loss:\",loss_per_epoch/num_batches)\n",
    "    saver.save(sess, model_dir + '/model')\n",
    "    print(\"Saved Model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# training accuracy\n",
    "all_predictions = []\n",
    "for j in range(num_batches):\n",
    "    path_dict = [path1_len[j*batch_size:(j+1)*batch_size], path2_len[j*batch_size:(j+1)*batch_size]]\n",
    "    word_dict = [word_p1_ids[j*batch_size:(j+1)*batch_size], word_p2_ids[j*batch_size:(j+1)*batch_size]]\n",
    "    pos_dict = [pos_p1_ids[j*batch_size:(j+1)*batch_size], pos_p2_ids[j*batch_size:(j+1)*batch_size]]\n",
    "    dep_dict = [dep_p1_ids[j*batch_size:(j+1)*batch_size], dep_p2_ids[j*batch_size:(j+1)*batch_size]]\n",
    "    y_dict = rel_ids[j*batch_size:(j+1)*batch_size]\n",
    "\n",
    "    feed_dict = {\n",
    "        path_length:path_dict,\n",
    "        word_ids:word_dict,\n",
    "        pos_ids:pos_dict,\n",
    "        dep_ids:dep_dict,\n",
    "        y:y_dict}\n",
    "    batch_predictions = sess.run(predictions, feed_dict)\n",
    "    all_predictions.append(batch_predictions)\n",
    "\n",
    "y_pred = []\n",
    "for i in range(num_batches):\n",
    "    for pred in all_predictions[i]:\n",
    "        y_pred.append(pred)\n",
    "\n",
    "count = 0\n",
    "for i in range(batch_size*num_batches):\n",
    "    count += y_pred[i]==rel_ids[i]\n",
    "accuracy = count/(batch_size*num_batches) * 100\n",
    "\n",
    "print(\"training accuracy\", accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = open(data_dir + '/test_lca_paths', 'rb')\n",
    "word_p, dep_p, pos_p = pickle.load(f)\n",
    "f.close()\n",
    "\n",
    "relations = []\n",
    "for line in open(data_dir + '/test_relations.txt'):\n",
    "    relations.append(line.strip().split()[0])\n",
    "\n",
    "length = len(word_p1)\n",
    "num_batches = int(length/batch_size)\n",
    "\n",
    "for i in range(length):\n",
    "    for j, word in enumerate(word_p[i]):\n",
    "        word = word.lower()\n",
    "        word_p[i][j] = word if word in word2id else unknown_token \n",
    "    for l, d in enumerate(dep_p[i]):\n",
    "        dep_p[i][l] = d if d in dep2id else 'OTH'\n",
    "        \n",
    "word_p_ids = np.ones([length, max_len_path],dtype=int)\n",
    "pos_p_ids = np.ones([length, max_len_path],dtype=int)\n",
    "dep_p_ids = np.ones([length, max_len_path],dtype=int)\n",
    "rel_ids = np.array([rel2id[rel] for rel in relations])\n",
    "path_len = np.array([len(w) for w in word_p], dtype=int)\n",
    "\n",
    "for i in range(length):\n",
    "    for j, w in enumerate(word_p[i]):\n",
    "        word_p_ids[i][j] = word2id[w]\n",
    "        \n",
    "    for j, w in enumerate(pos_p[i]):\n",
    "        pos_p_ids[i][j] = pos_tag(w)\n",
    "        \n",
    "    for j, w in enumerate(dep_p[i]):\n",
    "        dep_p_ids[i][j] = dep2id[w]\n",
    "\n",
    "# test predictions\n",
    "all_predictions = []\n",
    "for j in range(num_batches):\n",
    "    path_dict = [path1_len[j*batch_size:(j+1)*batch_size], path2_len[j*batch_size:(j+1)*batch_size]]\n",
    "    word_dict = [word_p1_ids[j*batch_size:(j+1)*batch_size], word_p2_ids[j*batch_size:(j+1)*batch_size]]\n",
    "    pos_dict = [pos_p1_ids[j*batch_size:(j+1)*batch_size], pos_p2_ids[j*batch_size:(j+1)*batch_size]]\n",
    "    dep_dict = [dep_p1_ids[j*batch_size:(j+1)*batch_size], dep_p2_ids[j*batch_size:(j+1)*batch_size]]\n",
    "    y_dict = rel_ids[j*batch_size:(j+1)*batch_size]\n",
    "\n",
    "    feed_dict = {\n",
    "        path_length:path_dict,\n",
    "        word_ids:word_dict,\n",
    "        pos_ids:pos_dict,\n",
    "        dep_ids:dep_dict,\n",
    "        y:y_dict}\n",
    "    batch_predictions = sess.run(predictions, feed_dict)\n",
    "    all_predictions.append(batch_predictions)\n",
    "\n",
    "y_pred = []\n",
    "for i in range(num_batches):\n",
    "    for pred in all_predictions[i]:\n",
    "        y_pred.append(pred)\n",
    "\n",
    "count = 0\n",
    "for i in range(batch_size*num_batches):\n",
    "    count += y_pred[i]==rel_ids[i]\n",
    "accuracy = count/(batch_size*num_batches) * 100\n",
    "\n",
    "print(\"test accuracy\", accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
